# coding=utf-8
"""
spider handler
Author: kaspar
Date: 2022-04-06
"""

import requests
import time
from concurrent.futures import ThreadPoolExecutor, as_completed
from bs4 import BeautifulSoup
from util.logger import Logger
from util.mongo import HandlerMongoDb
from util.config import Config

REQUEST_TIMEOUT = int(Config.get_conf('server', 'request_timeout'))
db_collection = Config.get_conf('db_mongo', 'db_collection')

BAD_WORDS = ['404', '403', 'Not Found', 'AV', '无码', '国产精品', '偷拍', '午夜精品', '毛片', '露脸', '建设中', '验证页面',
            '没有找到站点', '免费区', '域名售卖', '精品国产', '导航', '澳门', '黄片', '国产', 'not found', '域名出售', 'null',
             'Access Denied', 'Av', 'Not Acceptable', '未找到页面']

class WebCollector(object):
    """
    collector
    """
    def __init__(self):
        self.my_mongo = HandlerMongoDb()

    def my_spider_runonce(self):
        """
        爬虫主函数
        运行一次即可,用于第一次搜集网站数据
        """
        target_web = 'https://youquhome.com/page/{}/'
        scan_page = 1
        scan_cnt = 0
        insert_cnt = 0
        while True:
            page = requests.get(target_web.format(scan_page))
            if page.status_code == 404:
                break
            scan_cnt += 1
            soup = BeautifulSoup(page.content, 'html.parser')
            article_lists = soup.find_all('article')

            for article in article_lists:
                try:
                    url = article.contents[1].contents[-1].contents[1].contents[0]
                except Exception as e:
                    Logger.writelog('error', e)
                    continue
                if self.my_mongo.find_one(db_collection, {'url': url}) is None:
                    try:
                        res, page = WebCollector.check_url(url)
                        if res:
                            self.my_mongo.insert_one(db_collection, WebCollector.new_web_item(url, page))
                            insert_cnt += 1
                    except Exception as e:
                        Logger.writelog('error', 'request {}, {}'.format(url, e))
            scan_page += 1
        Logger.writelog('info', 'spider scan web {} finish.'.format(target_web))
        Logger.writelog('info', 'scan: web {}, insert: web {}'.format(scan_page, insert_cnt))

    def my_spider_runonce_1(self):
        """
        爬虫主函数
        运行一次即可,用于第一次搜集网站数据
        """
        target_web = 'https://www.9866.cn/index.php?page={}&c=&catid='
        scan_page = 1
        scan_cnt = 0
        insert_cnt = 0
        while True:
            page = requests.get(target_web.format(scan_page))
            if page.status_code == 404:
                break
            scan_cnt += 1
            soup = BeautifulSoup(page.content, 'html.parser')
            article_lists = soup.find_all('div', class_='col-sm-12 nopaddingleftright blog-list-item mb90')
            if len(article_lists) == 0:
                break
            for article in article_lists:
                try:
                    url = article.find_all('p', class_='mb20')[1].find_all('a')[0].text
                    if str(url).startswith('http'):
                        if self.my_mongo.find_one(db_collection, {'url': url}) is None:
                            try:
                                res, page = WebCollector.check_url(url)
                                if res:
                                    self.my_mongo.insert_one(db_collection, WebCollector.new_web_item(url, page))
                                    insert_cnt += 1
                            except Exception as e:
                                Logger.writelog('error', 'request {}, {}'.format(url, e))
                except Exception as e:
                    Logger.writelog('error', e)
                    continue
            scan_page += 1
        Logger.writelog('info', 'spider scan web {} finish.'.format(target_web))
        Logger.writelog('info', 'scan: web {}, insert: web {}'.format(scan_cnt, insert_cnt))

    def my_spider_daily(self):
        """
        定时爬虫
        检测首页，是否有新的文章
        """
        target_web = 'https://youquhome.com/'
        Logger.writelog('info', 'spider scan start.'.format(target_web))

        page = requests.get(target_web)
        if page.status_code == 404:
            Logger.writelog('error', '{} web is 404'.format(target_web))
            return

        soup = BeautifulSoup(page.content, 'html.parser')
        article_lists = soup.find_all('article')

        exist = 0
        insert_cnt = 0
        for article in article_lists:
            try:
                url = article.contents[1].contents[-1].contents[1].contents[0]
            except Exception as e:
                Logger.writelog('error', e)
                continue
            if self.my_mongo.find_one(db_collection, {'url': url}) is None:
                try:
                    res, page = WebCollector.check_url(url)
                    if res:
                        self.my_mongo.insert_one(db_collection, WebCollector.new_web_item(url, page))
                        insert_cnt += 1
                except Exception as e:
                    Logger.writelog('error', 'request {}, {}'.format(url, e))
            else:
                exist += 1
                if exist == 3:
                    break
        Logger.writelog('info', 'spider scan web {} finish.'.format(target_web))
        Logger.writelog('info', 'insert: web {}'.format(insert_cnt))

        self.my_spider_daily_1()

    def my_spider_daily_1(self):
        """
                定时爬虫
                检测首页，是否有新的文章
                """
        target_web = 'https://www.9866.cn'
        Logger.writelog('info', 'spider scan start.'.format(target_web))

        page = requests.get(target_web)
        if page.status_code == 404:
            Logger.writelog('error', '{} web is 404'.format(target_web))
            return

        soup = BeautifulSoup(page.content, 'html.parser')
        article_lists = soup.find_all('div', class_='col-sm-12 nopaddingleftright blog-list-item mb90')

        exist = 0
        insert_cnt = 0
        for article in article_lists:
            try:
                url = article.find_all('p', class_='mb20')[1].find_all('a')[0].text
            except Exception as e:
                Logger.writelog('error', e)
                continue

            if str(url).startswith('http'):
                if self.my_mongo.find_one(db_collection, {'url': url}) is None:
                    try:
                        res, page = WebCollector.check_url(url)
                        if res:
                            self.my_mongo.insert_one(db_collection, WebCollector.new_web_item(url, page))
                            insert_cnt += 1
                    except Exception as e:
                        Logger.writelog('error', 'request {}, {}'.format(url, e))
            else:
                exist += 1
                if exist == 3:
                    break
        Logger.writelog('info', 'spider scan web {} finish.'.format(target_web))
        Logger.writelog('info', 'insert: web {}'.format(insert_cnt))

    def manual_insert_urls(self, urls):
        """
        manual insert webs
        """
        assert isinstance(urls, list), "urls should be list"
        insert_cnt = 0
        for url in urls:
            if self.my_mongo.find_one(db_collection, {'url': url}) is None:
                try:
                    res, page = WebCollector.check_url(url)
                    if res:
                        self.my_mongo.insert_one(db_collection, WebCollector.new_web_item(url, page))
                        insert_cnt += 1
                except Exception as e:
                    Logger.writelog('error', 'request {}, {}'.format(url, e))
        Logger.writelog('info', 'total: {}, insert: {}'.format(len(urls), insert_cnt))

    @staticmethod
    def new_web_item(url, page):
        """
        gen mongo item
        return: dict
        """
        if HandlerMongoDb().find_one(db_collection, {'url': url}):
            raise Exception("web already exist.")
        try:
            soup = BeautifulSoup(page.content, 'html.parser')
            soup.prettify()
            new_web_item = {
                'url': url,
                'desp': soup.title.string,
                'timestamp': int(time.time())
            }
            return new_web_item
        except Exception as e:
            Logger.writelog('error', e)
            return None

    @staticmethod
    def check_bad_url(url, page):
        """
        检测违规或无效
        """
        soup = BeautifulSoup(page.content, 'html.parser')
        soup.prettify()
        url_title = str(soup.title.string)
        if all([url_title.find(bad_word) == -1 for bad_word in BAD_WORDS]):
            return True, page
        else:
            Logger.writelog('warning', 'not good url, bad words in it, {}_{}'
                            .format(url, url_title))
            return False, 'not good url, bad words in it.'

    @staticmethod
    def check_url(url):
        """
        check url
        True: good
        False: bad
        """
        try:
            page = requests.get(url, timeout=15)
            if page.elapsed.total_seconds() < REQUEST_TIMEOUT:
                return WebCollector.check_bad_url(url, page)
            else:
                Logger.writelog('info', 'not good url {}, request time over {}s. actually is {}s'
                                .format(url, REQUEST_TIMEOUT, page.elapsed.total_seconds()))
                return False, 'not good url, request too long.'
        except Exception as e:
            Logger.writelog('error', 'request {}, {}'.format(url, e))
            return False, 'not good url, request too long.'

class webCleaner():
    """
    clean database
    weekly
    """

    def __init__(self, SINGLE_THREAD = True):
        self._thread = SINGLE_THREAD
        self.my_mongo = HandlerMongoDb()

    def _checkone_health(self, url):
        """
        检测网站健康
        """
        Logger.writelog('info', 'check health. url {}'.format(url))
        try:
            res, page = WebCollector.check_url(url)
            if not res:
                self.my_mongo.delete_one(db_collection, {'url': url})
                Logger.writelog('warning', 'delete one web. url: {}.'.format(url))
                return False
            res, _ = WebCollector.check_bad_url(url, page)
            return res
        except Exception as e:
            Logger.writelog('error', 'request {}, {}'.format(url, e))
            self.my_mongo.delete_one(db_collection, {'url': url})
            Logger.writelog('warning', 'delete one web. url: {}'.format(url))
            return False

    def my_spider_weekly(self):
        """
        定时爬虫
        清理数据库，无效连接
        """
        Logger.writelog('info', 'clean web database start.')
        all_webs = self.my_mongo.find(db_collection, {'url': {'$exists': True}})
        delete_cnt = 0

        if self._thread:
            for web in all_webs:
                try:
                    res, page = WebCollector.check_url(web['url'])
                    if not res:
                        self.my_mongo.delete_one(db_collection, {'url': web['url']})
                        delete_cnt += 1
                        Logger.writelog('warning', 'delete one web. url: {}.'.format(web['url']))
                except Exception as e:
                    Logger.writelog('error', 'request {}, {}'.format(web['url'], e))
                    self.my_mongo.delete_one(db_collection, {'url': web['url']})
        else:
            clean_result = []
            with ThreadPoolExecutor(max_workers=4) as web_executor:
                task_webs = [web_executor.submit(self._checkone_health, web['url']) for web in all_webs]

                for _future in as_completed(task_webs):
                    clean_result.append(_future.result())
                delete_cnt = len([res for res in clean_result if not res])
        Logger.writelog('info', 'clean web database finish. delete web cnt {}.'.format(delete_cnt))


if __name__ == '__main__':
    # a = webCleaner(SINGLE_THREAD=False)
    # a.my_spider_weekly()
    b = WebCollector()
    # b.my_spider_daily()
    b.my_spider_daily_1()
    # b.my_spider_runonce_1()

    urls = ["http://www.ismycomputeron.com/",
      "http://imaninja.com/",
      "https://pointerpointer.com/",
      "http://corndogoncorndog.com/",
      "http://ihasabucket.com/",
      "http://www.nullingthevoid.com/",
      "http://www.muchbetterthanthis.com/",
      "http://www.yesnoif.com/",
      "http://iamawesome.com/",
      "http://www.pleaselike.com/",
      "https://crouton.net/",
      "https://corgiorgy.com/",
      "http://wutdafuk.com/",
      "http://unicodesnowmanforyou.com/",
      "http://www.crossdivisions.com/",
      "http://www.omfgdogs.com/",
      "https://oct82.com/",
      "http://chihuahuaspin.com/",
      "http://dogs.are.the.most.moe/",
      "http://www.blankwindows.com/",
      "http://www.trashloop.com/",
      "http://www.ascii-middle-finger.com/",
      "http://spaceis.cool/",
      "http://www.donothingfor2minutes.com/",
      "http://buildshruggie.com/",
      "http://buzzybuzz.biz/",
      "http://yeahlemons.com/",
      "http://burnie.com/",
      "http://wowenwilsonquiz.com/",
      "https://thepigeon.org/",
      "https://happyhappyhardcore.com/",
      "https://loopedforinfinity.com/",
      "http://greatbignothing.com/",
      "http://www.amialright.com/",
      "http://notdayoftheweek.com/",
      "https://alteredqualia.com/",
      "https://yesno.wtf/",
      "https://githubstat.us/",
      "http://most.moe//",
      "http://www.amialright.com/",
      "http://www.amialright.com/"]

    randomlinks = {}
    randomlinks[0] = "https://landscapes.littleworkshop.fr/"
    randomlinks[1] = "http://oos.moxiecode.com/js_webgl/half_spheres/"
    randomlinks[2] = "https://ambicular.com/"
    randomlinks[3] = "https://hostrider.com/"
    randomlinks[4] = "http://rainyscope.com/"
    randomlinks[5] = "http://treksit.com/"
    randomlinks[6] = "http://threejs.org/examples/webgl_shaders_ocean.html"
    randomlinks[7] = "http://mrdoob.neocities.org/022/"
    randomlinks[8] = "http://oos.moxiecode.com/js_webgl/spring/"
    randomlinks[9] = "https://larsberg.github.io/_lsr/2017/20160506_bubblesTwo/index.html"
    randomlinks[10] = "http://oos.moxiecode.com/js_webgl/particles_morph/"
    randomlinks[11] = "https://tabletopy.com/"
    randomlinks[12] = "http://oos.moxiecode.com/js_webgl/autumn/"
    randomlinks[13] = "https://larsberg.github.io/_lsr/2018/1014_laserBagPipe/index.html"
    randomlinks[14] = "https://larsberg.github.io/_lsr/2018/1016_longOne/index.html"
    randomlinks[15] = "http://robotics.ovh/"
    randomlinks[16] = "https://larsberg.github.io/_lsr/2018/1015_slugLuv/index.html"
    randomlinks[17] = "http://soulwire.github.io/Muscular-Hydrostats/"
    randomlinks[18] = "https://bongo.cat/"
    randomlinks[19] = "http://soulwire.github.io/Unwrapageddon/"
    randomlinks[20] = "http://asmrion.com/"
    randomlinks[21] = "http://ikeryou.jp/demoworks/tower/"
    randomlinks[22] = "https://larsberg.github.io/2018_nov_sketches/1010_ribWorm/index.html"
    randomlinks[23] = "https://codepen.io/ste-vg/full/BazEQbY"
    randomlinks[24] = "http://arodic.github.io/p/jellyfish/"
    randomlinks[25] = "https://larsberg.github.io/2018_nov_sketches/1004_gltf/index.html"
    randomlinks[26] = "http://virtocean.com"
    randomlinks[27] = "https://www.adultswim.com/etcetera/choir/"
    randomlinks[28] = "http://pluvior.com/"
    randomlinks[29] = "https://larsberg.github.io/_lsr/2017/2017/1007_thumptember/index.html"
    randomlinks[30] = "https://larsberg.github.io/2018_nov_sketches/1002_gltf/index.html"
    randomlinks[31] = "http://labs.fluuu.id/stars/"
    randomlinks[32] = "https://larsberg.github.io/_lsr/2017/2017/XmasXperiment/index.html"
    randomlinks[33] = "http://starrior.com/"
    randomlinks[34] = "http://rainbowhunt.com/"
    randomlinks[35] = "https://relaxcalm.netlify.app/"
    randomlinks[36] = "https://larsberg.github.io/_lsr/2017/20161129_puff/index.html"
    randomlinks[37] = "https://blocks.ovh/"
    randomlinks[38] = "https://larsberg.github.io/_lsr/2017/20161128_triboids/index.html"
    randomlinks[39] = "https://larsberg.github.io/_lsr/2017/20161127_skinDeep/index.html"
    randomlinks[40] = "https://larsberg.github.io/_lsr/2017/20161115_huckleberry/index.html"
    randomlinks[41] = "http://mrdoob.neocities.org/025/"
    randomlinks[42] = "https://larsberg.github.io/_lsr/2017/20161121_withSticks/index.html"
    randomlinks[43] = "https://stephaneginier.com/sculptgl/"
    randomlinks[44] = "https://larsberg.github.io/_lsr/2017/20161114_rollers/index.html"
    randomlinks[45] = "https://larsberg.github.io/_lsr/2017/20161112_emanation/index.html"
    randomlinks[46] = "https://larsberg.github.io/_lsr/2017/20161111_winding/index.html"
    randomlinks[47] = "https://ykob.github.io/sketch-threejs/sketch/instancing.html"
    randomlinks[48] = "http://soulwire.co.uk/experiments/recursion-toy/"
    randomlinks[49] = "http://2016.makemepulse.com/"
    randomlinks[50] = "https://larsberg.github.io/_lsr/2017/20161108_bronco/index.html"
    randomlinks[51] = "https://codepen.io/alexlesubmarine/full/aBzzRZ/"
    randomlinks[52] = "https://soulwire.co.uk/kinetic-canvas/"
    randomlinks[53] = "https://larsberg.github.io/_lsr/2017/20160315_jointz/index.html"
    randomlinks[54] = "https://larsberg.github.io/_lsr/2017/PaperLayers_rainbow/index.html"
    randomlinks[55] = "https://alexanderperrin.com.au/paper/shorttrip/"
    randomlinks[56] = "https://lab.colinpeyrat.com/dissolve/"
    randomlinks[57] = "https://larsberg.github.io/_lsr/2017/ldProjectionSketch/index.html"
    randomlinks[58] = "https://larsberg.github.io/_lsr/2017/20161107_dongles/index.html"
    randomlinks[59] = "https://ykob.github.io/sketch-threejs/sketch/aura.html"
    randomlinks[60] = "http://labs.fluuu.id/box-physics/"
    randomlinks[61] = "https://larsberg.github.io/_lsr/2017/ProjectionCubes/index.html"
    randomlinks[62] = "https://ykob.github.io/sketch-threejs/sketch/crystal.html"
    randomlinks[63] = "https://ykob.github.io/sketch-threejs/sketch/sun.html"
    randomlinks[64] = "http://slices.ovh/"
    randomlinks[65] = "https://ykob.github.io/sketch-threejs/sketch/newyear2019.html"
    randomlinks[66] = "https://defonic.com/"
    randomlinks[67] = "https://skuawk.com/"
    randomlinks[68] = "https://hipstersound.com/ambient.html"
    randomlinks[69] = "http://projects.defmech.com/codevember_2016/03/"
    randomlinks[70] = "http://codepen.io/cathbailh/full/pbXgkr/"
    randomlinks[71] = "http://www.staggeringbeauty.com/"
    randomlinks[72] = "https://mybearlove.com/"
    randomlinks[73] = "http://www.georgeandjonathan.com/#3"
    randomlinks[74] = "https://codepen.io/enxaneta/full/NqaBGy/"
    randomlinks[75] = "http://www.georgeandjonathan.com/#4"
    randomlinks[76] = "http://www.georgeandjonathan.com/#2"
    randomlinks[77] = "https://codepen.io/zadvorsky/full/PNXbGo/"
    randomlinks[78] = "https://ykob.github.io/sketch-threejs/sketch/planet.html"
    randomlinks[79] = "http://cabbi.bo/Bees/"
    randomlinks[80] = "https://codepen.io/gregh/full/zNzvOm/"
    randomlinks[81] = "https://ykob.github.io/sketch-threejs/sketch/webcam.html"
    randomlinks[82] = "https://ykob.github.io/sketch-threejs/sketch/flame.html"
    randomlinks[83] = "https://codepen.io/BenjiStander/full/Pzaggx/"
    randomlinks[84] = "https://demo.marpi.pl/eyes/"
    randomlinks[85] = "https://www.uberviz.io/viz/word-problems/"
    randomlinks[86] = "https://codepen.io/jcoulterdesign/full/xOOeMN/"
    randomlinks[87] = "https://ykob.github.io/sketch-threejs/sketch/attract.html"
    randomlinks[88] = "https://www.adultswim.com/etcetera/elastic-man/"
    randomlinks[89] = "https://scfex02.xl.digital/"
    randomlinks[90] = "https://ykob.github.io/sketch-threejs/sketch/transform.html"
    randomlinks[91] = "https://tabletopy.com/"
    randomlinks[92] = "https://hipstersound.com/rainy-screensaver.html"
    randomlinks[93] = "https://codepen.io/maneeshc/full/dXqdGz/"
    randomlinks[94] = "https://ykob.github.io/sketch-threejs/sketch/distort.html"
    randomlinks[95] = "https://codepen.io/Mombasa/full/leiGu/"
    randomlinks[96] = "https://ykob.github.io/sketch-threejs/sketch/image_data.html"
    randomlinks[97] = "https://ykob.github.io/sketch-threejs/sketch/comet.html"
    randomlinks[98] = "https://codepen.io/giana/full/LVvaNQ/"
    randomlinks[99] = "https://codepen.io/Yakudoo/full/oXJYxy/"
    randomlinks[100] = "https://ykob.github.io/sketch-threejs/sketch/fire_ball.html"
    randomlinks[101] = "https://island.goodboydigital.com/"
    randomlinks[102] = "https://dev.scottdarby.com/chaos-ink/"
    randomlinks[103] = "http://pluvior.com"
    randomlinks[104] = "https://www.hiteshsahu.com/MunichMetro"
    randomlinks[105] = "https://www.hiteshsahu.com/Snow"
    randomlinks[106] = "https://haxiomic.github.io/GPU-Fluid-Experiments/html5/"
    randomlinks[107] = "https://codepen.io/ScavengerFrontend/full/xxGwjox"
    randomlinks[108] = "https://www.adultswim.com/etcetera/soup/"
    randomlinks[109] = "https://paveldogreat.github.io/WebGL-Fluid-Simulation/"
    randomlinks[110] = "https://codepen.io/edball/full/GoYeQZ/"
    randomlinks[111] = "http://rugs.grindselect.com/"
    randomlinks[112] = "https://codepen.io/ykob/full/ojMwrv/"
    randomlinks[113] = "https://rainyscope.com/"
    randomlinks[114] = "http://mrdoob.com/files/temp/xplsv_obsidian/"
    randomlinks[115] = "https://sketches.vlucendo.com/cube/"
    randomlinks[116] = "https://codepen.io/shubniggurath/full/xxxQozp"
    randomlinks[117] = "http://ommexperience.com/"
    randomlinks[118] = "http://audiograph.xyz/"
    randomlinks[119] = "http://asmrion.com/"
    randomlinks[120] = "https://larsberg.github.io/2019sketchBook/sketches/13_pullmearound/build/"
    randomlinks[121] = "http://cabbi.bo/huldra/"
    randomlinks[122] = "http://touchpianist.com/"
    randomlinks[123] = "https://domenicobrz.github.io/webgl/projects/Volumetric%20art/"
    randomlinks[124] = "https://domenicobrz.github.io/webgl/projects/Sidescroller%20anim/"
    randomlinks[125] = "https://sombras.app"
    randomlinks[126] = "https://mrdoob.neocities.org/036/"
    randomlinks[127] = "https://domenicobrz.github.io/webgl/projects/wolf-header/"
    randomlinks[128] = "http://mrdoob.neocities.org/023/"
    randomlinks[129] = "https://codepen.io/tksiiii/full/VOEKLQ"
    randomlinks[130] = "http://magickeyboard.io/"
    randomlinks[131] = "http://calm.ovh/"
    randomlinks[132] = "http://mrdoob.neocities.org/020/"
    randomlinks[133] = "https://codepen.io/mephysto/full/yapYNX/"
    randomlinks[134] = "http://mrdoob.neocities.org/019/"
    randomlinks[135] = "https://codepen.io/al-ro/full/xxxxgxK"
    randomlinks[136] = "http://mrdoob.neocities.org/018/"
    randomlinks[137] = "https://domenicobrz.github.io/webgl/projects/gpu-fluid-sim/"
    randomlinks[138] = "http://mrdoob.neocities.org/016/"
    randomlinks[139] = "http://mrdoob.neocities.org/024/"
    randomlinks[140] = "https://domenicobrz.github.io/webgl/projects/marching%20cubes/"
    randomlinks[141] = "http://mrdoob.neocities.org/006/"
    randomlinks[142] = "http://mrdoob.neocities.org/015/"
    randomlinks[143] = "https://domenicobrz.github.io/webgl/projects/testportfolio/"
    randomlinks[144] = "https://domenicobrz.github.io/webgl/projects/Thomas%20de%20vos/"
    randomlinks[145] = "http://mrdoob.neocities.org/013/"
    randomlinks[146] = "https://codepen.io/iamjoshellis/full/KVdQqm/"
    randomlinks[147] = "http://mrdoob.neocities.org/012/"
    randomlinks[148] = "https://amsxyz.github.io/three-multifaceted-refraction/"
    randomlinks[149] = "http://mrdoob.neocities.org/007/"
    randomlinks[150] = "https://paveldogreat.github.io/WebGL-Fluid-Simulation/"
    randomlinks[151] = "https://ykob.github.io/sketch-threejs/sketch/instancing.html"
    randomlinks[152] = "http://mrdoob.neocities.org/004/"
    randomlinks[153] = "https://codepen.io/jhnsnc/full/GRJeOWZ"
    randomlinks[154] = "https://oimo.io/works/clock/"
    randomlinks[155] = "https://muda.co/collection/melt/"
    randomlinks[156] = "http://mrdoob.neocities.org/003/"
    randomlinks[157] = "https://robpayot.github.io/glowing-tree/build/"
    randomlinks[158] = "https://whiteflashwhitehit.com/content/2011/02/anisotropic_webgl.html"
    randomlinks[159] = "http://mrdoob.neocities.org/017/"
    randomlinks[160] = "https://codepen.io/shubniggurath/full/ERjXqy/"
    randomlinks[161] = "https://webglsamples.org/aquarium/aquarium.html"
    randomlinks[162] = "https://codepen.io/raurir/full/pjaEbj/"
    randomlinks[163] = "http://rainbowhunt.com"
    randomlinks[164] = "https://www.clicktorelease.com/code/io15/"
    randomlinks[165] = "https://tympanus.net/Tutorials/codrops-kinetic-typo/"
    randomlinks[166] = "https://codepen.io/chrisgannon/full/RwWVJWd"
    randomlinks[167] = "https://codepen.io/ycw/full/ExKyJdR"
    randomlinks[168] = "http://mrdoob.neocities.org/005/"
    randomlinks[169] = "https://r3f-bubbles.netlify.app/"
    randomlinks[170] = "https://domenicobrz.github.io/webgl/projects/SSRefractionDepthPeeling/"
    randomlinks[171] = "https://javier.xyz/pintr/"
    randomlinks[172] = "https://codepen.io/rocbear/full/rWdmWm/"
    randomlinks[173] = "http://wxs.ca/studies/001/"
    randomlinks[174] = "https://demo.marpi.pl/archan/eutow/"
    randomlinks[175] = "https://2019.makemepulse.com/"
    randomlinks[176] = "http://pluvior.com/"
    randomlinks[177] = "https://codepen.io/aderaaij/full/BapYONL"
    randomlinks[178] = "https://demo.marpi.pl/archan/aroha/"
    randomlinks[179] = "https://2018.makemepulse.com/"
    randomlinks[180] = "https://codepen.io/ycw/full/xxVPMwB"
    randomlinks[181] = "https://starrior.com/"
    randomlinks[182] = "https://mgz.me/?scene=CircuitGlass"
    randomlinks[183] = "https://sombras.app"
    randomlinks[184] = "http://ikeryou.jp/works/gobo/"
    randomlinks[185] = "https://codepen.io/Godje/full/RobJyX/"
    randomlinks[186] = "https://hipstersound.com/ambient.html"
    randomlinks[187] = "https://codepen.io/Mamboleoo/full/qqQbzv/"
    randomlinks[188] = "http://mrdoob.neocities.org/021/"
    randomlinks[189] = "https://classes.marpi.pl/camp/"
    randomlinks[190] = "https://codepen.io/seanfree/full/ZpYGJd/"
    randomlinks[191] = "https://codepen.io/cl4ws0n/full/YXXjWe/"
    randomlinks[192] = "http://guillaumegouessan.com/sketches/dancing-instances/"
    randomlinks[193] = "http://guillaumegouessan.com/sketches/ecailles/"
    randomlinks[194] = "http://guillaumegouessan.com/sketches/subsurface-scattering/"
    randomlinks[195] = "http://calm.ovh/"
    randomlinks[196] = "https://neal.fun/deep-sea"
    randomlinks[197] = "http://species-in-pieces.com/"
    randomlinks[198] = "https://christmasexperiments.com/2018/01/plume-and-the-forgotten-letter/"
    randomlinks[199] = "https://christmasexperiments.com/2018/02/louvre-2043/"
    randomlinks[200] = "https://threejs.org/examples/webgl_shaders_ocean.html"
    randomlinks[201] = "https://morphing.encharm.studio/"
    randomlinks[202] = "http://hostrider.com/"
    randomlinks[203] = "https://codepen.io/stivaliserna/full/GRNxGrR"
    randomlinks[204] = "https://www.vortex.dakumisu.fr/"
    randomlinks[205] = "http://stampede.it/"
    randomlinks[206] = "https://codepen.io/isladjan/full/zYqLxeG"
    randomlinks[207] = "https://mrdoob.neocities.org/037/"
    randomlinks[208] = "https://sciecode.github.io/soothe/"
    randomlinks[209] = "https://aidn.jp/twintail/"
    randomlinks[210] = "https://codepen.io/shubniggurath/full/GVQqjz"
    randomlinks[211] = "https://codepen.io/Yakudoo/full/rVGraP"
    randomlinks[212] = "https://codepen.io/Yakudoo/full/pgPgeb"
    randomlinks[213] = "https://codepen.io/Yakudoo/full/YGxYZj"
    randomlinks[214] = "https://codepen.io/Yakudoo/full/YXxmYR"
    randomlinks[215] = "https://codepen.io/chrisgannon/full/gNpWLZ"
    randomlinks[216] = "https://codepen.io/aaroniker/full/YoqNRB"
    randomlinks[217] = "https://codepen.io/jkantner/full/wLMoWx"
    randomlinks[218] = "https://artsandculture.google.com/experiment/blob-opera/AAHWrq360NcGbw"
    randomlinks[219] = "https://codepen.io/alexzaworski/full/xMvEVr"
    randomlinks[220] = "https://zoomquilt2.com/"
    randomlinks[221] = "https://calm.ovh/"
    randomlinks[222] = "http://robotics.ovh/"
    randomlinks[223] = "https://cdn.gitcdn.link/cdn/aatishb/drape/df7db9ef321b75b21bd01de2e977644d5666da9c/index.html"
    randomlinks[224] = "https://mrdoob.neocities.org/017/"
    randomlinks[225] = "http://blue.eastsidelag.com/"
    randomlinks[226] = "https://arkadia.xyz/"
    randomlinks[227] = "https://rainbowhunt.com/relaxing-raindrop-sounds.html"
    randomlinks[228] = "https://calming.netlify.app/"
    randomlinks[229] = "https://bubblespop.netlify.app/"
    randomlinks[230] = "https://physicsofbeauty.art"
    randomlinks[231] = "https://codepen.io/ge1doot/full/GgOYoy"
    randomlinks[232] = "http://eelslap.com/"
    randomlinks[233] = "https://smashthewalls.com/"
    randomlinks[234] = "http://www.koalastothemax.com/"
    randomlinks[235] = "https://defonic.com/"
    randomlinks[236] = "https://codepen.io/yuanchuan/full/yrobOe"
    randomlinks[237] = "http://codrops-svg.surge.sh/"
    randomlinks[238] = "https://mrdoob.neocities.org/035/"
    randomlinks[239] = "https://www.clicktorelease.com/code/codevember-2017/supershape/"
    randomlinks[240] = "https://landemy.netlify.app/"
    randomlinks[241] = "https://vaporwaves.netlify.app/"
    randomlinks[242] = "https://musiclab.chromeexperiments.com/Spectrogram/"
    randomlinks[243] = "https://musiclab.chromeexperiments.com/Rhythm/"
    randomlinks[244] = "https://musiclab.chromeexperiments.com/Shared-Piano/"
    randomlinks[245] = "https://musiclab.chromeexperiments.com/Arpeggios/"
    randomlinks[246] = "https://musiclab.chromeexperiments.com/Kandinsky/"
    randomlinks[247] = "https://codepen.io/yuanchuan/full/wZJqNK"
    randomlinks[248] = "https://tympanus.net/Development/ExplodingObjects/index-egg.html"
    randomlinks[249] = "https://tympanus.net/Development/ExplodingObjects/index-icosahedron.html"
    randomlinks[250] = "https://lumina.goodboydigital.com/"
    randomlinks[251] = "https://sketches.vlucendo.com/morphed-sphere/"
    randomlinks[252] = "https://sketches.vlucendo.com/portal/"
    randomlinks[253] = "https://codepen.io/shubniggurath/full/LaVmGP"
    randomlinks[254] = "https://codepen.io/soju22/full/ywLZPV"
    randomlinks[255] = "https://codepen.io/mwmwmw/full/wNWoMV"
    randomlinks[256] = "http://samsy.ninja/odd/index.html"
    randomlinks[257] = "https://codepen.io/Tibixx/full/mvLrqe"
    randomlinks[258] = "https://clickclickclick.click/"
    randomlinks[259] = "https://mgz.me/?scene=Patterns"
    randomlinks[260] = "https://mgz.me/?scene=TheLightCycles"
    randomlinks[261] = "https://mgz.me/?scene=TheCity"
    randomlinks[262] = "https://mgz.me/?scene=CosmicClouds"
    randomlinks[263] = "http://madebyevan.com/webgl-water/"
    randomlinks[264] = "https://tympanus.net/Tutorials/InteractiveParticles/"
    randomlinks[265] = "https://codepen.io/sdras/full/PVjPKa"
    randomlinks[266] = "https://codepen.io/DonKarlssonSan/full/ALyQBk/"
    randomlinks[267] = "https://2017.makemepulse.com/"
    randomlinks[268] = "https://codepen.io/tmrDevelops/full/gaRPbg/"
    randomlinks[269] = "https://tympanus.net/Tutorials/apple-fifth-avenue/?debug=true"
    randomlinks[270] = "https://tympanus.net/Development/MusicalInteractions/"
    randomlinks[271] = "http://ikeryou.jp/works/b/"
    randomlinks[272] = "https://massmigrations.com/agle"
    randomlinks[273] = "https://demo.marpi.pl/spider/index.html"
    randomlinks[274] = "https://odra.city/klessoi"
    randomlinks[275] = "https://demo.marpi.pl/explosion/"
    randomlinks[276] = "https://demo.marpi.pl/flower/"
    randomlinks[277] = "https://demo.marpi.pl/shattered_cube/"
    randomlinks[278] = "https://demo.marpi.pl/nefertiti/"
    randomlinks[279] = "https://classes.marpi.pl/camp/"
    randomlinks[280] = "https://codepen.io/xorxor_hu/full/vyXLJV/"
    randomlinks[281] = "http://mrdoob.neocities.org/008/"
    randomlinks[282] = "http://a-way-to-go.com/"
    randomlinks[283] = "https://origamisimulator.org/"
    randomlinks[284] = "https://demos.littleworkshop.fr/infinitown"
    randomlinks[285] = "https://threejs.org/examples/webgl_animation_keyframes"
    randomlinks[286] = "https://surface-floater.lusion.co/"
    randomlinks[287] = "https://threejs.org/examples/webgl_decals.html"
    randomlinks[288] = "https://threejs.org/examples/webgl_effects_anaglyph.html"
    randomlinks[289] = "https://threejs.org/examples/webgl_loader_gltf.html"
    randomlinks[290] = "https://demo.marpi.pl/ocean_desert/"
    randomlinks[291] = "https://codepen.io/Mamboleoo/full/GNvWwj/"
    randomlinks[292] = "https://aaron-bird.github.io/demo/rubiks-cube/"
    randomlinks[293] = "https://www.uberviz.io/viz/splice/"
    randomlinks[294] = "https://threejs.org/examples/webgl_materials_bumpmap.html"
    randomlinks[295] = "https://threejs.org/examples/webgl_materials_cubemap_dynamic.html"
    randomlinks[296] = "https://ykob.github.io/sketch-threejs/sketch/cyberspace.html"
    randomlinks[297] = "https://www.spielzeugz.de/html5/sticky-thing/"
    randomlinks[298] = "https://demos.littleworkshop.fr/track"
    randomlinks[299] = "http://www.dennis.video/"
    randomlinks[300] = "https://oguzeroglu.github.io/ROYGBIV/demo/villneuvedascq/application.html"
    randomlinks[301] = "https://david.li/paint/"
    randomlinks[302] = "https://hughsk.io/popcorn/"
    randomlinks[303] = "https://adventuremachine.4thfloorcreative.co.uk/"
    randomlinks[304] = "https://david.li/fluid/"
    randomlinks[305] = "https://david.li/flow/"
    randomlinks[306] = "https://www.hiteshsahu.com/SinkHole"
    randomlinks[307] = "https://codepen.io/iamjoshellis/full/qbBKZB/"
    randomlinks[308] = "https://codepen.io/xorxor_hu/full/woKBwo/"
    randomlinks[309] = "https://ykob.github.io/sketch-threejs/sketch/hole.html"
    randomlinks[310] = "https://codepen.io/frost084/full/QWbNyjN"
    randomlinks[311] = "https://codepen.io/Godje/full/mOzpEY/"
    randomlinks[312] = "https://yiwenl.github.io/Sketches/exps/67/"
    randomlinks[313] = "https://hextunel.netlify.app/"
    randomlinks[314] = "https://codepen.io/supah/full/XWdyLjb"
    randomlinks[315] = "https://codepen.io/nanoshrine/full/NWNazBL"
    randomlinks[316] = "https://5f6221884c2c4800dae91d92--threejslines.netlify.app/"
    randomlinks[317] = "https://adinunz.io/supershape/"
    randomlinks[318] = "http://codepen.io/bali_balo/full/yJOmgv/"
    randomlinks[319] = "https://alexanderperrin.com.au/triangles/ballooning/"
    randomlinks[320] = "https://prototypes.sennep.com/experiments/squeeshy/"
    randomlinks[321] = "https://boytchev.github.io/mannequin.js/examples/example-dynamic.html"
    randomlinks[322] = "https://raymarchingmatcaps.netlify.app/"
    randomlinks[323] = "https://boytchev.github.io/mannequin.js/examples/example-point.html"
    randomlinks[324] = "http://typedrummer.com/"
    randomlinks[325] = "https://christmasexperiments.com/2018/04/hatsumode/"
    randomlinks[326] = "https://christmasexperiments.com/2018/05/the-forest-spirits/"
    randomlinks[327] = "https://christmasexperiments.com/2018/06/xmas-tree/"
    randomlinks[328] = "https://christmasexperiments.com/2018/07/lithos/"
    randomlinks[329] = "https://christmasexperiments.com/2018/08/reflect/"
    randomlinks[330] = "https://rockpaperscissors.ai/"
    randomlinks[331] = "https://meowmart.mailchimp.com/"
    randomlinks[332] = "https://adinunz.io/translucentPearls/"
    randomlinks[333] = "https://ykob.github.io/sketch-threejs/sketch/fire_ball_2.html"
    randomlinks[334] = "https://busysimulator.com/"
    randomlinks[335] = "http://cabbibo.com/"
    randomlinks[336] = "https://musiclab.chromeexperiments.com/Song-Maker"
    randomlinks[337] = "http://www.edshark.com/ocean-of-words/"
    randomlinks[338] = "https://morse.withgoogle.com/learn/"
    randomlinks[339] = "https://www.patatap.com/"
    randomlinks[340] = "https://knots.netlify.app/"
    randomlinks[341] = "https://pudding.cool/2017/03/music-history/"
    randomlinks[342] = "https://www.micro-pano.com/pearl/index.html"
    randomlinks[343] = "https://synesthesia.rikard.io/"
    randomlinks[344] = "https://codepen.io/Hornebom/full/RjdyJr"
    randomlinks[345] = "https://codepen.io/Hornebom/full/oOavpd"
    randomlinks[346] = "https://codepen.io/cvaneenige/full/wNbRxW"
    randomlinks[347] = "http://2018.makemepulse.com/"
    randomlinks[348] = "https://codepen.io/hiteshsahu/full/jOqyPJG"
    randomlinks[349] = "https://www.yanlinma.com/flat-ui-and-a-half/"
    randomlinks[350] = "https://sokyokuban.com/"
    randomlinks[351] = "https://hostrider.com/"
    randomlinks[352] = "https://demo.marpi.pl/dark_haze/"
    randomlinks[353] = "https://codepen.io/elsemeow/full/gOwYMNR"
    randomlinks[354] = "https://codepen.io/tr13ze/full/PNWwVB"
    randomlinks[355] = "https://glitchy.website/"
    randomlinks[356] = "https://2020game.io/"
    randomlinks[357] = "https://codepen.io/miocene/full/rNOVWor"
    randomlinks[358] = "https://anengineer.netlify.app/"
    randomlinks[359] = "https://daftisland.netlify.app/"
    randomlinks[360] = "https://hipstersound.com/rainy-screensaver.html"
    randomlinks[361] = "https://vaporwaver.netlify.app/"
    randomlinks[362] = "https://www.clicktorelease.com/code/polygon-shredder/"
    randomlinks[363] = "https://www.clicktorelease.com/code/codevember-2017/chromeography/"
    randomlinks[364] = "https://www.clicktorelease.com/code/codevember-2017/glow-worms/"
    randomlinks[365] = "https://sequencer.henryfellerhoff.com/"
    randomlinks[366] = "https://rainbowhunt.com/relaxing-raindrop-sounds.html"
    randomlinks[367] = "https://virtocean.com"
    randomlinks[368] = "https://wetclap.com"
    randomlinks[369] = "https://slideventura.com/"
    randomlinks[370] = "https://recollection.ukon.dev/"
    randomlinks[371] = "https://codepen.io/amit_sheen/full/XWMXwvJ"
    randomlinks[372] = "https://www.sequencer64.com/"
    randomlinks[373] = "https://codepen.io/ste-vg/full/MWpxKYR"
    randomlinks[374] = "https://deardiary.ai/"
    randomlinks[375] = "https://threejs-audio-reactive-visual.netlify.app/"
    randomlinks[376] = "http://zoomquilt.org/"
    randomlinks[377] = "https://demo.marpi.pl/archan/black_rain/"
    randomlinks[378] = "https://flexbattle.netlify.app/"
    randomlinks[379] = "https://ykob.github.io/sketch-threejs/sketch/metal_cube.html"
    randomlinks[380] = "https://defonic.com/moodlight.html"

    # urls = [randomlinks[key] for key in randomlinks]
    # b.manual_insert_urls(urls)
